package com.gxljc.bear.crawler.itaogao.business21;

import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.commons.lang.ObjectUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;

import com.alibaba.fastjson.JSONObject;
import com.gxljc.commons.util.Pair;
import com.gxljc.commons.util.StringUtil;
import com.gxljc.bear.crawler.itaogao.NewsPageTable;
import com.gxljc.bear.crawler.itaogao.util.FetchDataUtil;
import com.gxljc.bear.crawler.itaogao.util.HbaseUtil;
import com.gxljc.bear.crawler.itaogao.util.HttpUtil;
import com.gxljc.bear.crawler.itaogao.util.ImageUtil;
import com.gxljc.bear.crawler.itaogao.util.MongodbUtil;
import com.gxljc.bear.crawler.util.DateUtil;

/**
 * @author haitao E-mail:haitao@summba.com
 * @version createTime：2021年9月18日 上午9:44:49
 * 
 */

public class Business21Crawler {
	public static final String BusinessUrl21 = "http://www.21cbr.com";
	private final static Logger LOG = Logger.getLogger(Business21Crawler.class);
	private final static String SOURCE = "21商业评论";
	private final static String MONGONAME_COLLECTION = "t_newspaper_business21";
	private List<String> startUrlList = new ArrayList<String>();
	private List<String> detailUrlList = new ArrayList<String>();
	private Map<String, Object> newsMap = new HashMap<String, Object>();
	protected boolean proxy;
	
	public Business21Crawler() {
		newsMap.put("status", 0);
		newsMap.put("mediaName", SOURCE);
		proxy = false;
	}

	public void startCrawler() {
		// 获取初始导向url
		try {
			LOG.info("=====start crawl initialization url======");
			genStartUrl();
			LOG.info(String.format("=====finished crawl initialization url====== initialization url size is %s ",
					startUrlList.size()));
		} catch (Exception e) {
			e.printStackTrace();
		}

		// 获取指定期数url
		genDetailUrl();
		LOG.info(String.format("=====finished crawl detail url====== detail url size is %s ", detailUrlList.size()));

		// 抓取详细页面html,并 写入mongodb,hbase中
		crawlerDetail();

	}

	@SuppressWarnings("unchecked")
	public void genStartUrl() throws Exception {
		List<String> startUrl = new ArrayList<String>();
		// 查询2021年以后的有效url,检索至2100年
		for (int i = 2016; i <= 2100; i++) {
			String startCrawlerUrl = BusinessUrl21 + "/html/magzine/" + String.valueOf(i) + "/";
			if (HttpUtil.checkUrlexists(startCrawlerUrl)) {
				startUrl.add(startCrawlerUrl);
			} else {
				break;
			}
		}
		for (String stUrl : startUrl) {
			Pair<String, byte[]> pair = FetchDataUtil.fetResult(stUrl, proxy);
			if(pair == null)continue;
			String result = "";
			result = pair.first;
			if (result != null) {
				JSONObject jsonResult = (JSONObject) JSONObject.parse(result);
				if (jsonResult != null) {
					// add header
					for (String url : (List<String>) jsonResult.get("crawlUrl")) {
						startUrlList.add(BusinessUrl21 + url);
					}
				}
			}
		}
	}

	@SuppressWarnings("unchecked")
	public void genDetailUrl() {
		if (startUrlList.size() > 0) {
			for (String stUrl : startUrlList) {
				Pair<String, byte[]> pair = FetchDataUtil.fetResult(stUrl, proxy);
				if (pair == null)continue;
				String result = "";
				result = pair.first;
				if (result != null) {
					JSONObject jsonResult = (JSONObject) JSONObject.parse(result);
					if (jsonResult != null) {
						// add header
						for (String url : (List<String>) jsonResult.get("detailUrl")) {
							String crawlerUrl = BusinessUrl21 + url;
							// 需要重爬才加入爬虫url队列
							if (!MongodbUtil.needReCrawl(crawlerUrl,MONGONAME_COLLECTION)) {
								detailUrlList.add(BusinessUrl21 + url);
							}
						}
					}
				}
			}

		}
	}

	@SuppressWarnings("unchecked")
	public void crawlerDetail() {
		int countUpdate = 0;
		if (detailUrlList.size() > 0) {
			for (String detailUrl : detailUrlList) {
				LOG.info(String.format("insert the detailUrl is %s ", detailUrl));
				String result = "";
				Pair<String, byte[]> pair = FetchDataUtil.fetResult(detailUrl.trim(), proxy);
				if(pair == null)continue;
				NewsPageTable table = new NewsPageTable();
				result = pair.first;
				byte[] html = pair.second;
				if (result != null) {
					JSONObject jsonResult = (JSONObject) JSONObject.parse(result);
					if (jsonResult != null) {
						List<String> contentList = new ArrayList<String>();
						List<String> tagNameList = new ArrayList<String>();
						Object content = jsonResult.get("content");
						if (content != null) {
							contentList = (List<String>) content;
						}
						if (contentList.size() == 0) {
							continue;
						}
						Object tagName = jsonResult.get("tagName");
						if (tagName != null) {
							tagNameList = (List<String>) tagName;
						}
						if (tagNameList.size() != 0) {
							newsMap.put("tagName", tagNameList);
						}
						String id = ObjectUtils.toString(StringUtil.genMD5Val(detailUrl));
						String title = ObjectUtils.toString(jsonResult.get("title"));
						String subTitle = ObjectUtils.toString(jsonResult.get("subTitle"));
						String logoUrl = ObjectUtils.toString(jsonResult.get("logoUrl"));
						if (!logoUrl.isEmpty()) {
							logoUrl = BusinessUrl21 + logoUrl;
						}
						
						// 添加images字段
						Object images = jsonResult.get("images");
						List<String> imagesPlatform = new ArrayList<String>();
						if (images != null) {
							List<String> imagesList = new ArrayList<String>();
							for(String image:(List<String>) images){
								image = BusinessUrl21 +image;
								imagesList.add(image);
							}
							try {
								imagesPlatform = ImageUtil.imageConvertplatform(imagesList);
							} catch (Exception e) {
								LOG.info("images convert Error！");
								// e.printStackTrace();
							}
						}
						if (imagesPlatform.size() != 0) {
							newsMap.put("images", imagesPlatform);
						}
						
						// 添加基础数据
						newsMap.put("_id", id);
						newsMap.put("title", title);
						newsMap.put("subTitle", subTitle);
						newsMap.put("logoUrl", logoUrl);
						newsMap.put("crawlDate", new Date());
						newsMap.put("crawlUrl", detailUrl);
						newsMap.put("content", StringUtils.join(contentList, "####").replace("　　####", "").trim());

						// 添加内容和发布时间
						handleAuthorTime(jsonResult.get("author_time"));

						// 添加标志
						newsMap.put("crawlFlag", 1);

						// 初始化table值
						table.setTitle(title);
						table.setId(id);

						// 转换logoUrl字段
						try {
							ImageUtil.imageConvertplatform(logoUrl);
						} catch (Exception e) {
							LOG.info(String.format("logoUrl convert error!error url is %s", logoUrl));
							e.printStackTrace();
						}

						// 更新到mongodb库
						MongodbUtil.update2Mongodb(newsMap, MONGONAME_COLLECTION);//update2Mongodb(newsMap);

						// 更新到hbase中
						HbaseUtil.saveHbase(table, html);
						countUpdate += 1;
					}
				}
			}
			LOG.info(String.format("all successed update id size is %s ", countUpdate));
		}
		LOG.info("all detailUrl size=" + detailUrlList.size());
	}

	@SuppressWarnings("unchecked")
	public void crawlerDetailSingle(String url) {
		if (!url.isEmpty()) {
			String result = "";
			NewsPageTable table = new NewsPageTable();
			Pair<String, byte[]> pair = FetchDataUtil.fetResult(url.trim(), proxy);
			if(pair == null){
				return;
			}
			result = pair.first;
			byte[] html = pair.second;
			System.out.println("haitao");
			if (result != null) {
				JSONObject jsonResult = (JSONObject) JSONObject.parse(result);
				if (jsonResult != null) {
					List<String> contentList = (List<String>) jsonResult.get("content");
					String id = ObjectUtils.toString(StringUtil.genMD5Val(url));
					String title = ObjectUtils.toString(jsonResult.get("title"));
					String subTitle = ObjectUtils.toString(jsonResult.get("subTitle"));
					String logoUrl = ObjectUtils.toString(jsonResult.get("logoUrl"));
					if (!logoUrl.isEmpty()) {
						logoUrl = BusinessUrl21 + logoUrl;
					}
					
					// 添加images字段
					Object images = jsonResult.get("images");
					List<String> imagesPlatform = new ArrayList<String>();
					if (images != null) {
						List<String> imagesList = new ArrayList<String>();
						for (String image : (List<String>) images) {
							image = BusinessUrl21 + image;
							imagesList.add(image);
						}
						try {
							imagesPlatform = ImageUtil.imageConvertplatform(imagesList);
						} catch (Exception e) {
							LOG.info("images convert Error！");
							// e.printStackTrace();
						}
					}
					if (imagesPlatform.size() != 0) {
						newsMap.put("images", imagesPlatform);
					}
					
					// 添加基础数据
					newsMap.put("_id", id);
					newsMap.put("title", title);
					newsMap.put("subTitle", subTitle);
					newsMap.put("logoUrl", logoUrl);
					newsMap.put("crawlDate", new Date());
					newsMap.put("crawlUrl", url);
					newsMap.put("content", StringUtils.join(contentList, "####").replace("　　####", "").trim());

					// 添加内容和发布时间
					handleAuthorTime(jsonResult.get("author_time"));
					System.out.println("================");
					// 添加标志
					newsMap.put("crawlFlag", 1);

					// 初始化table值
					table.setTitle(title);
					table.setId(id);
					System.out.println("**************");
					// 转换logoUrl字段
					try {
						ImageUtil.imageConvertplatform(logoUrl);
					} catch (Exception e) {
						LOG.info(String.format("logoUrl convert error!error url is %s", logoUrl));
						e.printStackTrace();
					}
					System.out.println("=======logoUrl============");
					System.out.println(newsMap);

					// 更新到mongodb库
					MongodbUtil.update2Mongodb(newsMap, MONGONAME_COLLECTION);

					// 更新到hbase中
					HbaseUtil.saveHbase(table, html);

					LOG.info(String.format("insert the id/rowkey is %s ", id));
				}
			}
		}

	}

	public void handleAuthorTime(Object author_time) {
		String author = "";
		String publishDate = "";
		String source = "";
		String authorTime = ObjectUtils.toString(author_time);
		String[] authorTimeSplit = new String[] {};
		if (!authorTime.isEmpty()) {
			authorTimeSplit = authorTime.split("作者:|作者：");
			String[] sourceSplit = authorTimeSplit[0].split("来源:|来源：");
			String[] auTime = authorTimeSplit[1].split("发布时间：|发布时间：");
			if (sourceSplit.length == 2) {
				source = sourceSplit[1];
			}
			if (auTime.length == 2) {
				author = auTime[0].trim();
				publishDate = auTime[1];
			} else if (auTime.length == 1 && !auTime[0].contains("-")) {
				author = auTime[0].trim();
			} else if (auTime.length == 1 && auTime[0].split("-").length == 3) {
				publishDate = auTime[0];
			}
		}
		Date pd = DateUtil.parseFormat(publishDate, "yy-MM-dd HH:mm");
		newsMap.put("author", author);
		if (pd != null) {
			newsMap.put("publishDate", pd);
		}
		newsMap.put("source", source);
	}
	
	

}
